{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "https://video.pearvideo.com/mp4/third/20191203/cont-1628826-10037564-152133-hd.mp4\n",
    "https://video.pearvideo.com/mp4/adshort/20191203/cont-1628718-14656106_adpkg-ad_hd.mp4\n",
    "https://video.pearvideo.com/mp4/adshort/20191202/cont-1628390-14653009_adpkg-ad_hd.mp4\n",
    "Accept-Encoding: identity;q=1, *;q=0\n",
    "DNT: 1\n",
    "Range: bytes=0-\n",
    "Referer: https://www.pearvideo.com/video_1628826\n",
    "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\n",
    "\"\"\"\n",
    "# url = 'https://video.pearvideo.com/mp4/adshort/20191203/cont-1628718-14656106_adpkg-ad_hd.mp4'\n",
    "# media = requests.get(url=url,headers=headers).content\n",
    "# with open('m.mp4', 'wb') as fp:\n",
    "#     fp.write(media)\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "视频\\PS教程如何快速准确的校准白平衡.mp4 获取成功\n",
      "视频\\PS教程如何快速准确的校准白平衡.mp4 保存成功\n",
      "视频\\以前打电话为什么要先转动拨号盘.mp4 获取成功\n",
      "视频\\以前打电话为什么要先转动拨号盘.mp4 保存成功\n",
      "视频\\PS教程如何快速准确的校准白平衡.mp4 获取成功\n",
      "视频\\PS教程如何快速准确的校准白平衡.mp4 保存成功\n",
      "视频\\罗永浩再创业，叫卖是鲨纹儿童书包.mp4 获取成功\n",
      "视频\\罗永浩再创业，叫卖是鲨纹儿童书包.mp4 保存成功\n",
      "视频\\转基因黄金大米有望2021年开始种植.mp4 获取成功\n",
      "视频\\转基因黄金大米有望2021年开始种植.mp4 保存成功\n",
      "视频\\人类宜居的系外行星在哪.mp4 获取成功\n",
      "视频\\人类宜居的系外行星在哪.mp4 保存成功\n",
      "视频\\保质期一年的苹果在美国上市.mp4 获取成功\n",
      "视频\\保质期一年的苹果在美国上市.mp4 保存成功\n",
      "视频\\第四季度苹果市场份额将超越华为.mp4 获取成功\n",
      "视频\\第四季度苹果市场份额将超越华为.mp4 保存成功\n",
      "视频\\残疾人的智能假肢到底有多智能.mp4 获取成功\n",
      "视频\\残疾人的智能假肢到底有多智能.mp4 保存成功\n",
      "视频\\谁保护你在网络上的钱.mp4 获取成功\n",
      "视频\\谁保护你在网络上的钱.mp4 保存成功\n",
      "视频\\华为高管2020年鸿蒙将超Linux.mp4 获取成功\n",
      "视频\\华为高管2020年鸿蒙将超Linux.mp4 保存成功\n",
      "视频\\第189课mocha跟踪和替换平面曲线.mp4 获取成功\n",
      "视频\\第189课mocha跟踪和替换平面曲线.mp4 保存成功\n",
      "视频\\比飞机还快的汽车，每小时1678公里.mp4 获取成功\n",
      "视频\\比飞机还快的汽车，每小时1678公里.mp4 保存成功\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "from multiprocessing.dummy import Pool\n",
    "from lxml import etree\n",
    "import requests\n",
    "\n",
    "\n",
    "url = 'https://www.pearvideo.com/category_8' # 要爬取的分类页面\n",
    "video_base_url = 'https://www.pearvideo.com/'\n",
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'\n",
    "}\n",
    "\n",
    "# 定义线程池\n",
    "download_pool = Pool(5)\n",
    "save_pool = Pool(5)\n",
    "\n",
    "# 创建保存视频的目录\n",
    "video_category = '视频'\n",
    "if not os.path.exists(video_category):\n",
    "    os.mkdir('视频')\n",
    "\n",
    "\n",
    "# 主页面  \n",
    "page_text = requests.get(url=url, headers = headers).text\n",
    "tree = etree.HTML(page_text)\n",
    "title_list = tree.xpath('//div[@class=\"vervideo-title\"]//text()') # 标题列表\n",
    "video_suffix_list = tree.xpath('//div[@class=\"vervideo-bd\"]/a/@href') # 标题url后缀列表 video_1628826\n",
    "\n",
    "\n",
    "# real_url_list = [] # 获取到的视频地址列表\n",
    "# file_name_list = [] # 文件名列表\n",
    "file_url = []\n",
    "for video_suffix in video_suffix_list:\n",
    "    video_url = video_base_url + video_suffix # 视频详情页 https://www.pearvideo.com/video_1629045，从中提取视频的url\n",
    "    # 请求详情页\n",
    "    video_detail_text = requests.get(url=video_url, headers=headers).text\n",
    "    v_tree = etree.HTML(video_detail_text)\n",
    "    title = v_tree.xpath('//div[@class=\"video-tt-box\"]//h1//text()')[0] +'.mp4' # 罗永浩再创业，叫卖是鲨纹儿童书包\n",
    "    title = title.replace('？','').replace('：','') # 替换掉文件名中特殊符号\n",
    "    file_name = os.path.join(video_category,title)\n",
    "    pattern = 'srcUrl=\"(.*?)\",vdoUrl' # 从页面js中匹配真实的视频url\n",
    "    real_video_url = re.findall(pattern, video_detail_text, re.S)[0] # srcUrl=\"https://video.pearvideo.com/mp4/adshort/20191203/cont-1629045-14659742_adpkg-ad_hd.mp4\",vdoUrl\n",
    "    file_url.append({'name':file_name, 'url':real_video_url}) # 将文件名，url对应存入字典\n",
    "\n",
    "def downloader(file_url):\n",
    "    video_content = requests.get(url=file_url['url'], headers=headers).content # 视频内容\n",
    "#     print(file_url['name'],'获取成功')\n",
    "    with open(file_url['name'], 'wb') as fp:\n",
    "        fp.write(video_content)\n",
    "#         print(file_url['name'],'保存成功')\n",
    "   \n",
    "run = download_pool.map(downloader, file_url)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
